{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model with Metrics\n",
    "\n",
    "Example testing a model with custom metrics.\n",
    "\n",
    "Metrics can be \n",
    "\n",
    "  * A ```COUNTER``` : the returned value will increment the current value\n",
    "  * A ```GAUGE``` : the returned value will overwrite the current value\n",
    "  * A ```TIMER``` : a number of millisecs. Prometheus SUM and COUNT metrics will be created.\n",
    "  \n",
    "You need to provide a list of dictionaries each with the following:\n",
    "\n",
    "  * a ```type``` : COUNTER, GAUGE, or TIMER\n",
    "  * a ```key``` : a user defined key\n",
    "  * a ```value``` : a float value\n",
    "  \n",
    "See example code below:\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mclass\u001b[39;49;00m \u001b[04m\u001b[32mModelWithMetrics\u001b[39;49;00m(\u001b[36mobject\u001b[39;49;00m):\r\n",
      "\r\n",
      "    \u001b[34mdef\u001b[39;49;00m \u001b[32m__init__\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m):\r\n",
      "        \u001b[34mprint\u001b[39;49;00m(\u001b[33m\"\u001b[39;49;00m\u001b[33mInitialising\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\r\n",
      "\r\n",
      "    \u001b[34mdef\u001b[39;49;00m \u001b[32mpredict\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m,X,features_names):\r\n",
      "        \u001b[34mprint\u001b[39;49;00m(\u001b[33m\"\u001b[39;49;00m\u001b[33mPredict called\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\r\n",
      "        \u001b[34mreturn\u001b[39;49;00m X\r\n",
      "\r\n",
      "    \u001b[34mdef\u001b[39;49;00m \u001b[32mmetrics\u001b[39;49;00m():\r\n",
      "        \u001b[34mreturn\u001b[39;49;00m [\r\n",
      "            {\u001b[33m\"\u001b[39;49;00m\u001b[33mtype\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[33m\"\u001b[39;49;00m\u001b[33mCOUNTER\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m,\u001b[33m\"\u001b[39;49;00m\u001b[33mkey\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[33m\"\u001b[39;49;00m\u001b[33mmycounter\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m,\u001b[33m\"\u001b[39;49;00m\u001b[33mvalue\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[34m1\u001b[39;49;00m}, \u001b[37m# a counter which will increase by the given value\u001b[39;49;00m\r\n",
      "            {\u001b[33m\"\u001b[39;49;00m\u001b[33mtype\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[33m\"\u001b[39;49;00m\u001b[33mGAUGE\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m,\u001b[33m\"\u001b[39;49;00m\u001b[33mkey\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[33m\"\u001b[39;49;00m\u001b[33mmygauge\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m,\u001b[33m\"\u001b[39;49;00m\u001b[33mvalue\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[34m100\u001b[39;49;00m}, \u001b[37m# a gauge which will be set to given value\u001b[39;49;00m\r\n",
      "            {\u001b[33m\"\u001b[39;49;00m\u001b[33mtype\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[33m\"\u001b[39;49;00m\u001b[33mTIMER\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m,\u001b[33m\"\u001b[39;49;00m\u001b[33mkey\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[33m\"\u001b[39;49;00m\u001b[33mmytimer\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m,\u001b[33m\"\u001b[39;49;00m\u001b[33mvalue\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\u001b[34m20.2\u001b[39;49;00m}, \u001b[37m# a timer which will add sum and count metrics - assumed millisecs\u001b[39;49;00m\r\n",
      "            ]\r\n",
      "    \r\n"
     ]
    }
   ],
   "source": [
    "!pygmentize ModelWithMetrics.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# REST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Installing application source...\n",
      "Build completed successfully\n"
     ]
    }
   ],
   "source": [
    "!s2i build -E environment_rest . seldonio/seldon-core-s2i-python3:0.3-SNAPSHOT model-with-metrics-rest:0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "672480d40f662cc3d37d999264ea3ecca234eb9bb0cbd685772f1f5a8e515240\r\n"
     ]
    }
   ],
   "source": [
    "!docker run --name \"model-with-metrics\" -d --rm -p 5000:5000 model-with-metrics-rest:0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rm -f proto/prediction*.py\r\n",
      "rm -f proto/prediction.proto\r\n",
      "rm -rf proto/__pycache__\r\n",
      "rm -f fbs/*.py\r\n",
      "rm -rf fbs/__pycache__\r\n",
      "cp ../../proto/prediction.proto ./proto\r\n",
      "python -m grpc.tools.protoc -I. --python_out=. --grpc_python_out=. ./proto/prediction.proto\r\n"
     ]
    }
   ],
   "source": [
    "!cd ../../../wrappers/testing && make build_protos"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\r\n",
      "SENDING NEW REQUEST:\r\n",
      "{'meta': {}, 'data': {'names': ['sepal_length', 'sepal_width', 'petal_length', 'petal_width'], 'ndarray': [[6.668, 4.529, 4.791, 0.649]]}}\r\n",
      "RECEIVED RESPONSE:\r\n",
      "{'data': {'names': ['t:0', 't:1', 't:2', 't:3'], 'ndarray': [[6.668, 4.529, 4.791, 0.649]]}, 'meta': {'metrics': [{'key': 'mycounter', 'type': 'COUNTER', 'value': 1}, {'key': 'mygauge', 'type': 'GAUGE', 'value': 100}, {'key': 'mytimer', 'type': 'TIMER', 'value': 20.2}]}}\r\n",
      "\r\n",
      "Time 0.00577998161315918\r\n"
     ]
    }
   ],
   "source": [
    "!python ../../../wrappers/testing/tester.py contract.json 0.0.0.0 5000 -p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model-with-metrics\r\n"
     ]
    }
   ],
   "source": [
    "!docker rm model-with-metrics --force"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# gRPC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Installing application source...\n",
      "Build completed successfully\n"
     ]
    }
   ],
   "source": [
    "!s2i build -E environment_grpc . seldonio/seldon-core-s2i-python3:0.3-SNAPSHOT model-with-metrics-grpc:0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b83028c73787596d2a641c84eedc2173951ad6746d78897e459173ce4799a521\r\n"
     ]
    }
   ],
   "source": [
    "!docker run --name \"model-with-metrics\" -d --rm -p 5000:5000 model-with-metrics-grpc:0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rm -f proto/prediction*.py\r\n",
      "rm -f proto/prediction.proto\r\n",
      "rm -rf proto/__pycache__\r\n",
      "rm -f fbs/*.py\r\n",
      "rm -rf fbs/__pycache__\r\n",
      "cp ../../proto/prediction.proto ./proto\r\n",
      "python -m grpc.tools.protoc -I. --python_out=. --grpc_python_out=. ./proto/prediction.proto\r\n"
     ]
    }
   ],
   "source": [
    "!cd ../../../wrappers/testing && make build_protos"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\r\n",
      "SENDING NEW REQUEST:\r\n",
      "data {\r\n",
      "  names: \"sepal_length\"\r\n",
      "  names: \"sepal_width\"\r\n",
      "  names: \"petal_length\"\r\n",
      "  names: \"petal_width\"\r\n",
      "  ndarray {\r\n",
      "    values {\r\n",
      "      list_value {\r\n",
      "        values {\r\n",
      "          number_value: 5.681\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 4.435\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 6.005\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 0.73\r\n",
      "        }\r\n",
      "      }\r\n",
      "    }\r\n",
      "  }\r\n",
      "}\r\n",
      "\r\n",
      "RECEIVED RESPONSE:\r\n",
      "meta {\r\n",
      "  metrics {\r\n",
      "    key: \"mycounter\"\r\n",
      "    value: 1.0\r\n",
      "  }\r\n",
      "  metrics {\r\n",
      "    key: \"mygauge\"\r\n",
      "    type: GAUGE\r\n",
      "    value: 100.0\r\n",
      "  }\r\n",
      "  metrics {\r\n",
      "    key: \"mytimer\"\r\n",
      "    type: TIMER\r\n",
      "    value: 20.200000762939453\r\n",
      "  }\r\n",
      "}\r\n",
      "data {\r\n",
      "  names: \"t:0\"\r\n",
      "  names: \"t:1\"\r\n",
      "  names: \"t:2\"\r\n",
      "  names: \"t:3\"\r\n",
      "  ndarray {\r\n",
      "    values {\r\n",
      "      list_value {\r\n",
      "        values {\r\n",
      "          number_value: 5.681\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 4.435\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 6.005\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 0.73\r\n",
      "        }\r\n",
      "      }\r\n",
      "    }\r\n",
      "  }\r\n",
      "}\r\n",
      "\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "!python ../../../wrappers/testing/tester.py contract.json 0.0.0.0 5000 -p --grpc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model-with-metrics\r\n"
     ]
    }
   ],
   "source": [
    "!docker rm model-with-metrics --force"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test using Minikube\n",
    "\n",
    "**Due to a [minikube/s2i issue](https://github.com/SeldonIO/seldon-core/issues/253) you will need Minikube version 0.25.2**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There is a newer version of minikube available (v0.30.0).  Download it here:\n",
      "https://github.com/kubernetes/minikube/releases/tag/v0.30.0\n",
      "\n",
      "To disable this notification, run the following:\n",
      "minikube config set WantUpdateNotification false\n",
      "Starting local Kubernetes v1.9.4 cluster...\n",
      "Starting VM...\n",
      "Getting VM IP address...\n",
      "Moving files into cluster...\n",
      "Setting up certs...\n",
      "Connecting to cluster...\n",
      "Setting up kubeconfig...\n",
      "Starting cluster components...\n",
      "Kubectl is now configured to use the cluster.\n",
      "Loading cached images from config file.\n"
     ]
    }
   ],
   "source": [
    "!minikube start --vm-driver kvm2 --memory 4096 --feature-gates=CustomResourceValidation=true --extra-config=apiserver.Authorization.Mode=RBAC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clusterrolebinding.rbac.authorization.k8s.io/kube-system-cluster-admin created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create clusterrolebinding kube-system-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "$HELM_HOME has been configured at /home/clive/.helm.\n",
      "\n",
      "Tiller (the Helm server-side component) has been installed into your Kubernetes Cluster.\n",
      "\n",
      "Please note: by default, Tiller is deployed with an insecure 'allow unauthenticated users' policy.\n",
      "To prevent this, run `helm init` with the --tiller-tls-verify flag.\n",
      "For more information on securing your installation see: https://docs.helm.sh/using_helm/#securing-your-helm-installation\n",
      "Happy Helming!\n"
     ]
    }
   ],
   "source": [
    "!helm init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"tiller-deploy\" rollout to finish: 0 of 1 updated replicas are available...\n",
      "deployment \"tiller-deploy\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/tiller-deploy -n kube-system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   seldon-core-crd\n",
      "LAST DEPLOYED: Sat Nov  3 08:25:19 2018\n",
      "NAMESPACE: default\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/ServiceAccount\n",
      "NAME                        SECRETS  AGE\n",
      "seldon-spartakus-volunteer  1        0s\n",
      "\n",
      "==> v1beta1/ClusterRole\n",
      "NAME                        AGE\n",
      "seldon-spartakus-volunteer  0s\n",
      "\n",
      "==> v1beta1/ClusterRoleBinding\n",
      "NAME                        AGE\n",
      "seldon-spartakus-volunteer  0s\n",
      "\n",
      "==> v1/ConfigMap\n",
      "NAME                     DATA  AGE\n",
      "seldon-spartakus-config  3     1s\n",
      "\n",
      "==> v1beta1/CustomResourceDefinition\n",
      "NAME                                         AGE\n",
      "seldondeployments.machinelearning.seldon.io  0s\n",
      "\n",
      "==> v1beta1/Deployment\n",
      "NAME                        DESIRED  CURRENT  UP-TO-DATE  AVAILABLE  AGE\n",
      "seldon-spartakus-volunteer  1        0        0           0          0s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "NOTES: TODO\n",
      "\n",
      "\n",
      "NAME:   seldon-core\n",
      "LAST DEPLOYED: Sat Nov  3 08:25:20 2018\n",
      "NAMESPACE: default\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/ClusterRoleBinding\n",
      "NAME            AGE\n",
      "seldon-default  0s\n",
      "\n",
      "==> v1beta1/Role\n",
      "NAME          AGE\n",
      "seldon-local  0s\n",
      "\n",
      "==> v1/RoleBinding\n",
      "NAME    AGE\n",
      "seldon  0s\n",
      "\n",
      "==> v1/Service\n",
      "NAME                          TYPE       CLUSTER-IP     EXTERNAL-IP  PORT(S)                        AGE\n",
      "seldon-core-seldon-apiserver  NodePort   10.108.30.143  <none>       8080:30088/TCP,5000:31980/TCP  0s\n",
      "seldon-core-redis             ClusterIP  10.109.3.187   <none>       6379/TCP                       0s\n",
      "\n",
      "==> v1beta1/Deployment\n",
      "NAME                                DESIRED  CURRENT  UP-TO-DATE  AVAILABLE  AGE\n",
      "seldon-core-seldon-apiserver        1        1        1           0          0s\n",
      "seldon-core-seldon-cluster-manager  1        1        1           0          0s\n",
      "seldon-core-redis                   1        1        1           0          0s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                                                 READY  STATUS             RESTARTS  AGE\n",
      "seldon-core-seldon-apiserver-5976c46469-tq2l8        0/1    ContainerCreating  0         0s\n",
      "seldon-core-seldon-cluster-manager-55db6c898c-p6mk8  0/1    ContainerCreating  0         0s\n",
      "seldon-core-redis-98c4948f7-gm5sv                    0/1    ContainerCreating  0         0s\n",
      "\n",
      "==> v1/ServiceAccount\n",
      "NAME    SECRETS  AGE\n",
      "seldon  1        0s\n",
      "\n",
      "==> v1beta1/ClusterRole\n",
      "NAME                AGE\n",
      "seldon-crd-default  0s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "Thank you for installing Seldon Core.\n",
      "\n",
      "Documentation can be found at https://github.com/SeldonIO/seldon-core\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install ../../../helm-charts/seldon-core-crd --name seldon-core-crd  --set usage_metrics.enabled=true\n",
    "!helm install ../../../helm-charts/seldon-core --name seldon-core "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   seldon-core-analytics\n",
      "LAST DEPLOYED: Thu Nov  8 13:40:09 2018\n",
      "NAMESPACE: seldon\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/Secret\n",
      "NAME                 TYPE    DATA  AGE\n",
      "grafana-prom-secret  Opaque  1     0s\n",
      "\n",
      "==> v1/ConfigMap\n",
      "NAME                       DATA  AGE\n",
      "alertmanager-server-conf   1     0s\n",
      "grafana-import-dashboards  7     0s\n",
      "prometheus-rules           4     0s\n",
      "prometheus-server-conf     1     0s\n",
      "\n",
      "==> v1/ServiceAccount\n",
      "NAME        SECRETS  AGE\n",
      "prometheus  1        0s\n",
      "\n",
      "==> v1beta1/ClusterRole\n",
      "NAME        AGE\n",
      "prometheus  0s\n",
      "\n",
      "==> v1beta1/ClusterRoleBinding\n",
      "NAME        AGE\n",
      "prometheus  0s\n",
      "\n",
      "==> v1/Job\n",
      "NAME                            DESIRED  SUCCESSFUL  AGE\n",
      "grafana-prom-import-dashboards  1        0           0s\n",
      "\n",
      "==> v1beta1/Deployment\n",
      "NAME                     DESIRED  CURRENT  UP-TO-DATE  AVAILABLE  AGE\n",
      "alertmanager-deployment  1        1        1           0          0s\n",
      "grafana-prom-deployment  1        1        1           0          0s\n",
      "prometheus-deployment    1        1        1           0          0s\n",
      "\n",
      "==> v1beta1/DaemonSet\n",
      "NAME                      DESIRED  CURRENT  READY  UP-TO-DATE  AVAILABLE  NODE SELECTOR  AGE\n",
      "prometheus-node-exporter  1        1        0      1           0          <none>         0s\n",
      "\n",
      "==> v1/Service\n",
      "NAME                      TYPE       CLUSTER-IP      EXTERNAL-IP  PORT(S)       AGE\n",
      "alertmanager              ClusterIP  10.102.58.24    <none>       80/TCP        0s\n",
      "grafana-prom              NodePort   10.100.218.244  <none>       80:30688/TCP  0s\n",
      "prometheus-node-exporter  ClusterIP  None            <none>       9100/TCP      0s\n",
      "prometheus-seldon         ClusterIP  10.110.100.130  <none>       80/TCP        0s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                                      READY  STATUS             RESTARTS  AGE\n",
      "grafana-prom-import-dashboards-t52ts      0/1    ContainerCreating  0         0s\n",
      "alertmanager-deployment-7fbfdfdfb6-4q7hz  0/1    ContainerCreating  0         0s\n",
      "grafana-prom-deployment-7b45fb85d4-7znsq  0/1    ContainerCreating  0         0s\n",
      "prometheus-node-exporter-p4bwc            0/1    ContainerCreating  0         0s\n",
      "prometheus-deployment-cbfd78cc7-nnjd6     0/1    ContainerCreating  0         0s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "NOTES: TODO\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install seldon-core-analytics --name seldon-core-analytics --set grafana_prom_admin_password=password --set persistence.enabled=false --repo https://storage.googleapis.com/seldon-charts "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " * Port forward the dashboard when running\n",
    "  ```\n",
    "     kubectl port-forward $(kubectl get pods -n seldon -l app=grafana-prom-server -o jsonpath='{.items[0].metadata.name}') -n seldon 3000:3000\n",
    "  ```\n",
    "  * Visit http://localhost:3000/dashboard/db/prediction-analytics?refresh=5s&orgId=1 and login using \"admin\" and the password you set above when launching with helm."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# REST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I1108 13:41:46.852606    1559 build.go:50] Running S2I version \"v1.1.12\"\n",
      "I1108 13:41:46.852742    1559 util.go:58] Getting docker credentials for seldonio/seldon-core-s2i-python3:0.3-SNAPSHOT\n",
      "I1108 13:41:46.852762    1559 util.go:74] Using  credentials for pulling seldonio/seldon-core-s2i-python3:0.3-SNAPSHOT\n",
      "I1108 13:41:46.882915    1559 docker.go:487] Using locally available image \"seldonio/seldon-core-s2i-python3:0.3-SNAPSHOT\"\n",
      "I1108 13:41:46.884452    1559 build.go:163] \n",
      "Builder Image:\t\t\tseldonio/seldon-core-s2i-python3:0.3-SNAPSHOT\n",
      "Source:\t\t\t\t.\n",
      "Output Image Tag:\t\tmodel-with-metrics-rest:0.1\n",
      "Environment:\t\t\tMODEL_NAME=ModelWithMetrics,API_TYPE=REST,SERVICE_TYPE=MODEL,PERSISTENCE=0\n",
      "Environment File:\t\tenvironment_rest\n",
      "Labels:\t\t\t\t\n",
      "Incremental Build:\t\tdisabled\n",
      "Remove Old Build:\t\tdisabled\n",
      "Builder Pull Policy:\t\tif-not-present\n",
      "Previous Image Pull Policy:\tif-not-present\n",
      "Quiet:\t\t\t\tdisabled\n",
      "Layered Build:\t\t\tdisabled\n",
      "Docker Endpoint:\t\ttcp://192.168.39.50:2376\n",
      "Docker Pull Config:\t\t/home/clive/.docker/config.json\n",
      "Docker Pull User:\t\tcliveseldon\n",
      "\n",
      "I1108 13:41:46.885812    1559 docker.go:487] Using locally available image \"seldonio/seldon-core-s2i-python3:0.3-SNAPSHOT\"\n",
      "I1108 13:41:46.889703    1559 docker.go:487] Using locally available image \"seldonio/seldon-core-s2i-python3:0.3-SNAPSHOT\"\n",
      "I1108 13:41:46.889713    1559 docker.go:718] Image sha256:29d7be19772bfb306b12e4ece8ba007330c72a5c2c574ba3769ba69ad06ea898 contains io.openshift.s2i.scripts-url set to \"image:///s2i/bin\"\n",
      "I1108 13:41:46.889939    1559 scm.go:20] DownloadForSource .\n",
      "I1108 13:41:46.889999    1559 sti.go:204] Preparing to build model-with-metrics-rest:0.1\n",
      "I1108 13:41:46.890159    1559 download.go:38] Copying sources from \".\" to \"/tmp/s2i141179583/upload/src\"\n",
      "I1108 13:41:46.890265    1559 fs.go:260] F \"contract.json\" -> \"/tmp/s2i141179583/upload/src/contract.json\"\n",
      "I1108 13:41:46.890325    1559 fs.go:260] F \"deployment-rest.json\" -> \"/tmp/s2i141179583/upload/src/deployment-rest.json\"\n",
      "I1108 13:41:46.890358    1559 fs.go:260] F \"modelWithMetrics.ipynb\" -> \"/tmp/s2i141179583/upload/src/modelWithMetrics.ipynb\"\n",
      "I1108 13:41:46.890427    1559 fs.go:260] F \"environment_rest\" -> \"/tmp/s2i141179583/upload/src/environment_rest\"\n",
      "I1108 13:41:46.890459    1559 fs.go:260] F \"environment_rest~\" -> \"/tmp/s2i141179583/upload/src/environment_rest~\"\n",
      "I1108 13:41:46.890492    1559 fs.go:260] F \"ModelWithMetrics.py~\" -> \"/tmp/s2i141179583/upload/src/ModelWithMetrics.py~\"\n",
      "I1108 13:41:46.890525    1559 fs.go:260] F \"deployment-grpc.json\" -> \"/tmp/s2i141179583/upload/src/deployment-grpc.json\"\n",
      "I1108 13:41:46.891110    1559 fs.go:260] F \"deployment-rest.json~\" -> \"/tmp/s2i141179583/upload/src/deployment-rest.json~\"\n",
      "I1108 13:41:46.891147    1559 fs.go:247] D \".ipynb_checkpoints\" -> \"/tmp/s2i141179583/upload/src/.ipynb_checkpoints\"\n",
      "I1108 13:41:46.891200    1559 fs.go:260] F \".ipynb_checkpoints/modelWithMetrics-checkpoint.ipynb\" -> \"/tmp/s2i141179583/upload/src/.ipynb_checkpoints/modelWithMetrics-checkpoint.ipynb\"\n",
      "I1108 13:41:46.891271    1559 fs.go:260] F \"environment_grpc~\" -> \"/tmp/s2i141179583/upload/src/environment_grpc~\"\n",
      "I1108 13:41:46.891303    1559 fs.go:260] F \"ModelWithMetrics.py\" -> \"/tmp/s2i141179583/upload/src/ModelWithMetrics.py\"\n",
      "I1108 13:41:46.891334    1559 fs.go:260] F \"environment_grpc\" -> \"/tmp/s2i141179583/upload/src/environment_grpc\"\n",
      "I1108 13:41:46.891381    1559 fs.go:260] F \"deployment-grpc.json~\" -> \"/tmp/s2i141179583/upload/src/deployment-grpc.json~\"\n",
      "I1108 13:41:46.891416    1559 install.go:261] Using \"assemble\" installed from \"image:///s2i/bin/assemble\"\n",
      "I1108 13:41:46.891446    1559 install.go:261] Using \"run\" installed from \"image:///s2i/bin/run\"\n",
      "I1108 13:41:46.891457    1559 install.go:261] Using \"save-artifacts\" installed from \"image:///s2i/bin/save-artifacts\"\n",
      "I1108 13:41:46.891504    1559 ignore.go:64] .s2iignore file does not exist\n",
      "I1108 13:41:46.891510    1559 sti.go:213] Clean build will be performed\n",
      "I1108 13:41:46.891515    1559 sti.go:216] Performing source build from .\n",
      "I1108 13:41:46.891532    1559 sti.go:227] Running \"assemble\" in \"model-with-metrics-rest:0.1\"\n",
      "I1108 13:41:46.891536    1559 sti.go:573] Using image name seldonio/seldon-core-s2i-python3:0.3-SNAPSHOT\n",
      "I1108 13:41:46.892846    1559 docker.go:487] Using locally available image \"seldonio/seldon-core-s2i-python3:0.3-SNAPSHOT\"\n",
      "I1108 13:41:46.892879    1559 sti.go:453] No user environment provided (no environment file found in application sources)\n",
      "I1108 13:41:46.892909    1559 sti.go:691] starting the source uploading ...\n",
      "I1108 13:41:46.892920    1559 tar.go:217] Adding \"/tmp/s2i141179583/upload\" to tar ...\n",
      "I1108 13:41:46.892967    1559 tar.go:312] Adding to tar: /tmp/s2i141179583/upload/scripts as scripts\n",
      "I1108 13:41:46.895541    1559 docker.go:718] Image sha256:29d7be19772bfb306b12e4ece8ba007330c72a5c2c574ba3769ba69ad06ea898 contains io.openshift.s2i.scripts-url set to \"image:///s2i/bin\"\n",
      "I1108 13:41:46.895550    1559 docker.go:793] Base directory for S2I scripts is '/s2i/bin'. Untarring destination is '/tmp'.\n",
      "I1108 13:41:46.895558    1559 docker.go:949] Setting \"/bin/sh -c tar -C /tmp -xf - && /s2i/bin/assemble\" command for container ...\n",
      "I1108 13:41:46.895631    1559 docker.go:958] Creating container with options {Name:\"s2i_seldonio_seldon_core_s2i_python3_0_3_SNAPSHOT_beeacde3\" Config:{Hostname: Domainname: User: AttachStdin:false AttachStdout:true AttachStderr:false ExposedPorts:map[] Tty:false OpenStdin:true StdinOnce:true Env:[MODEL_NAME=ModelWithMetrics API_TYPE=REST SERVICE_TYPE=MODEL PERSISTENCE=0] Cmd:[/bin/sh -c tar -C /tmp -xf - && /s2i/bin/assemble] Healthcheck:<nil> ArgsEscaped:false Image:seldonio/seldon-core-s2i-python3:0.3-SNAPSHOT Volumes:map[] WorkingDir: Entrypoint:[] NetworkDisabled:false MacAddress: OnBuild:[] Labels:map[] StopSignal: StopTimeout:<nil> Shell:[]} HostConfig:&{Binds:[] ContainerIDFile: LogConfig:{Type: Config:map[]} NetworkMode: PortBindings:map[] RestartPolicy:{Name: MaximumRetryCount:0} AutoRemove:false VolumeDriver: VolumesFrom:[] CapAdd:[] CapDrop:[] DNS:[] DNSOptions:[] DNSSearch:[] ExtraHosts:[] GroupAdd:[] IpcMode: Cgroup: Links:[] OomScoreAdj:0 PidMode: Privileged:false PublishAllPorts:false ReadonlyRootfs:false SecurityOpt:[] StorageOpt:map[] Tmpfs:map[] UTSMode: UsernsMode: ShmSize:67108864 Sysctls:map[] Runtime: ConsoleSize:[0 0] Isolation: Resources:{CPUShares:0 Memory:0 NanoCPUs:0 CgroupParent: BlkioWeight:0 BlkioWeightDevice:[] BlkioDeviceReadBps:[] BlkioDeviceWriteBps:[] BlkioDeviceReadIOps:[] BlkioDeviceWriteIOps:[] CPUPeriod:0 CPUQuota:0 CPURealtimePeriod:0 CPURealtimeRuntime:0 CpusetCpus: CpusetMems: Devices:[] DeviceCgroupRules:[] DiskQuota:0 KernelMemory:0 MemoryReservation:0 MemorySwap:0 MemorySwappiness:<nil> OomKillDisable:<nil> PidsLimit:0 Ulimits:[] CPUCount:0 CPUPercent:0 IOMaximumIOps:0 IOMaximumBandwidth:0} Mounts:[] Init:<nil>}} ...\n",
      "I1108 13:41:46.929308    1559 docker.go:990] Attaching to container \"d661ca52e1d7662ef0badcb2be3688f90759c9756498c0aee3a0089e2a99190b\" ...\n",
      "I1108 13:41:46.944928    1559 docker.go:1001] Starting container \"d661ca52e1d7662ef0badcb2be3688f90759c9756498c0aee3a0089e2a99190b\" ...\n",
      "I1108 13:41:47.121961    1559 tar.go:312] Adding to tar: /tmp/s2i141179583/upload/src as src\n",
      "I1108 13:41:47.122034    1559 tar.go:312] Adding to tar: /tmp/s2i141179583/upload/src/.ipynb_checkpoints as src/.ipynb_checkpoints\n",
      "I1108 13:41:47.122107    1559 tar.go:312] Adding to tar: /tmp/s2i141179583/upload/src/.ipynb_checkpoints/modelWithMetrics-checkpoint.ipynb as src/.ipynb_checkpoints/modelWithMetrics-checkpoint.ipynb\n",
      "I1108 13:41:47.122217    1559 tar.go:312] Adding to tar: /tmp/s2i141179583/upload/src/ModelWithMetrics.py as src/ModelWithMetrics.py\n",
      "I1108 13:41:47.122267    1559 tar.go:312] Adding to tar: /tmp/s2i141179583/upload/src/ModelWithMetrics.py~ as src/ModelWithMetrics.py~\n",
      "I1108 13:41:47.122311    1559 tar.go:312] Adding to tar: /tmp/s2i141179583/upload/src/contract.json as src/contract.json\n",
      "I1108 13:41:47.122355    1559 tar.go:312] Adding to tar: /tmp/s2i141179583/upload/src/deployment-grpc.json as src/deployment-grpc.json\n",
      "I1108 13:41:47.122395    1559 tar.go:312] Adding to tar: /tmp/s2i141179583/upload/src/deployment-grpc.json~ as src/deployment-grpc.json~\n",
      "I1108 13:41:47.122506    1559 tar.go:312] Adding to tar: /tmp/s2i141179583/upload/src/deployment-rest.json as src/deployment-rest.json\n",
      "I1108 13:41:47.122627    1559 tar.go:312] Adding to tar: /tmp/s2i141179583/upload/src/deployment-rest.json~ as src/deployment-rest.json~\n",
      "I1108 13:41:47.122690    1559 tar.go:312] Adding to tar: /tmp/s2i141179583/upload/src/environment_grpc as src/environment_grpc\n",
      "I1108 13:41:47.122890    1559 tar.go:312] Adding to tar: /tmp/s2i141179583/upload/src/environment_grpc~ as src/environment_grpc~\n",
      "I1108 13:41:47.122940    1559 tar.go:312] Adding to tar: /tmp/s2i141179583/upload/src/environment_rest as src/environment_rest\n",
      "I1108 13:41:47.123039    1559 tar.go:312] Adding to tar: /tmp/s2i141179583/upload/src/environment_rest~ as src/environment_rest~\n",
      "I1108 13:41:47.123120    1559 tar.go:312] Adding to tar: /tmp/s2i141179583/upload/src/modelWithMetrics.ipynb as src/modelWithMetrics.ipynb\n",
      "I1108 13:41:47.129342    1559 sti.go:699] ---> Installing application source...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I1108 13:41:47.157389    1559 docker.go:1032] Waiting for container \"d661ca52e1d7662ef0badcb2be3688f90759c9756498c0aee3a0089e2a99190b\" to stop ...\n",
      "I1108 13:41:47.244738    1559 docker.go:1057] Invoking PostExecute function\n",
      "I1108 13:41:47.244769    1559 postexecutorstep.go:68] Skipping step: store previous image\n",
      "I1108 13:41:47.244773    1559 postexecutorstep.go:117] Executing step: commit image\n",
      "I1108 13:41:47.246352    1559 postexecutorstep.go:522] Checking for new Labels to apply... \n",
      "I1108 13:41:47.246362    1559 postexecutorstep.go:530] Creating the download path '/tmp/s2i141179583/metadata'\n",
      "I1108 13:41:47.246431    1559 postexecutorstep.go:464] Downloading file \"/tmp/.s2i/image_metadata.json\"\n",
      "I1108 13:41:47.284759    1559 postexecutorstep.go:538] unable to download and extract 'image_metadata.json' ... continuing\n",
      "I1108 13:41:47.287415    1559 docker.go:1091] Committing container with dockerOpts: {Reference:model-with-metrics-rest:0.1 Comment: Author: Changes:[] Pause:false Config:0xc420092c80}, config: {Hostname: Domainname: User: AttachStdin:false AttachStdout:false AttachStderr:false ExposedPorts:map[] Tty:false OpenStdin:false StdinOnce:false Env:[MODEL_NAME=ModelWithMetrics API_TYPE=REST SERVICE_TYPE=MODEL PERSISTENCE=0] Cmd:[/s2i/bin/run] Healthcheck:<nil> ArgsEscaped:false Image: Volumes:map[] WorkingDir: Entrypoint:[] NetworkDisabled:false MacAddress: OnBuild:[] Labels:map[io.openshift.s2i.scripts-url:image:///s2i/bin io.openshift.s2i.build.source-location:. io.k8s.display-name:model-with-metrics-rest:0.1 io.openshift.s2i.build.image:seldonio/seldon-core-s2i-python3:0.3-SNAPSHOT] StopSignal: StopTimeout:<nil> Shell:[]}\n",
      "I1108 13:41:47.356945    1559 postexecutorstep.go:392] Executing step: report success\n",
      "I1108 13:41:47.356968    1559 postexecutorstep.go:397] Successfully built model-with-metrics-rest:0.1\n",
      "I1108 13:41:47.356978    1559 postexecutorstep.go:93] Skipping step: remove previous image\n",
      "I1108 13:41:47.357049    1559 docker.go:968] Removing container \"d661ca52e1d7662ef0badcb2be3688f90759c9756498c0aee3a0089e2a99190b\" ...\n",
      "I1108 13:41:47.380567    1559 docker.go:978] Removed container \"d661ca52e1d7662ef0badcb2be3688f90759c9756498c0aee3a0089e2a99190b\"\n",
      "I1108 13:41:47.380704    1559 cleanup.go:33] Removing temporary directory /tmp/s2i141179583\n",
      "I1108 13:41:47.380723    1559 fs.go:302] Removing directory '/tmp/s2i141179583'\n",
      "I1108 13:41:47.381363    1559 build.go:175] Build completed successfully\n"
     ]
    }
   ],
   "source": [
    "!eval $(minikube docker-env) && s2i build -E environment_rest . seldonio/seldon-core-s2i-python3:0.3-SNAPSHOT model-with-metrics-rest:0.1 --loglevel 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldondeployment.machinelearning.seldon.io/mymodel created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create -f deployment-rest.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wait until ready (replicas == replicasAvailable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "map[predictorStatus:[map[name:mymodel-mymodel-svc-orch replicas:1 replicasAvailable:1] map[replicasAvailable:1 name:mymodel-mymodel-complex-model-0 replicas:1]] state:Available]"
     ]
    }
   ],
   "source": [
    "!kubectl get seldondeployments mymodel -o jsonpath='{.status}' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rm -f proto/prediction*.py\r\n",
      "rm -f proto/prediction.proto\r\n",
      "rm -rf proto/__pycache__\r\n",
      "mkdir -p ./proto\r\n",
      "touch ./proto/__init__.py\r\n",
      "cp ../../proto/prediction.proto ./proto\r\n",
      "python -m grpc.tools.protoc -I. --python_out=. --grpc_python_out=. ./proto/prediction.proto\r\n"
     ]
    }
   ],
   "source": [
    "!cd ../../../util/api_tester && make build_protos "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "SENDING NEW REQUEST:\n",
      "{'meta': {}, 'data': {'names': ['sepal_length', 'sepal_width', 'petal_length', 'petal_width'], 'ndarray': [[7.974098303518131, 3.149273146516985, 8.219107590512102, 2.3086360840551796]]}}\n",
      "Getting token from http://192.168.39.50:31307/oauth/token\n",
      "{\"access_token\":\"343409a6-0abd-4b8c-ad83-dd1f9657fef4\",\"token_type\":\"bearer\",\"expires_in\":42651,\"scope\":\"read write\"}\n",
      "RECEIVED RESPONSE:\n",
      "{'meta': {'puid': 'l05rsshl95btcv6rvr2k6ig3oa', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'model-with-metrics-rest:0.1'}, 'metrics': [{'key': 'mycounter', 'type': 'COUNTER', 'value': 1.0}, {'key': 'mygauge', 'type': 'GAUGE', 'value': 100.0}, {'key': 'mytimer', 'type': 'TIMER', 'value': 20.2}]}, 'data': {'names': ['t:0', 't:1', 't:2', 't:3'], 'ndarray': [[7.974098303518131, 3.149273146516985, 8.219107590512102, 2.3086360840551796]]}}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!python ../../../util/api_tester/api-tester.py contract.json \\\n",
    "    `minikube ip` `kubectl get svc -l app=seldon-apiserver-container-app -o jsonpath='{.items[0].spec.ports[0].nodePort}'` \\\n",
    "    --oauth-key oauth-key --oauth-secret oauth-secret -p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldondeployment.machinelearning.seldon.io \"mymodel\" deleted\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl delete -f deployment-rest.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# gRPC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Installing application source...\n",
      "Build completed successfully\n"
     ]
    }
   ],
   "source": [
    "!eval $(minikube docker-env) && s2i build -E environment_grpc . seldonio/seldon-core-s2i-python3:0.3-SNAPSHOT model-with-metrics-grpc:0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldondeployment.machinelearning.seldon.io/mymodel created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create -f deployment-grpc.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wait until ready (replicas == replicasAvailable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "map[predictorStatus:[map[replicasAvailable:1 name:mymodel-mymodel-svc-orch replicas:1] map[name:mymodel-mymodel-complex-model-0 replicas:1 replicasAvailable:1]] state:Available]"
     ]
    }
   ],
   "source": [
    "!kubectl get seldondeployments mymodel -o jsonpath='{.status}' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rm -f proto/prediction*.py\r\n",
      "rm -f proto/prediction.proto\r\n",
      "rm -rf proto/__pycache__\r\n",
      "mkdir -p ./proto\r\n",
      "touch ./proto/__init__.py\r\n",
      "cp ../../proto/prediction.proto ./proto\r\n",
      "python -m grpc.tools.protoc -I. --python_out=. --grpc_python_out=. ./proto/prediction.proto\r\n"
     ]
    }
   ],
   "source": [
    "!cd ../../../util/api_tester && make build_protos "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validate on Grafana\n",
    "\n",
    "To check the metrics have appeared on Prometheus and are available in Grafana you could create a new graph in a dashboard and use the query:\n",
    "\n",
    "```\n",
    "mycounter_total\n",
    "```\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "SENDING NEW REQUEST:\n",
      "data {\n",
      "  names: \"sepal_length\"\n",
      "  names: \"sepal_width\"\n",
      "  names: \"petal_length\"\n",
      "  names: \"petal_width\"\n",
      "  ndarray {\n",
      "    values {\n",
      "      list_value {\n",
      "        values {\n",
      "          number_value: 5.018607555163812\n",
      "        }\n",
      "        values {\n",
      "          number_value: 4.150787936306038\n",
      "        }\n",
      "        values {\n",
      "          number_value: 9.145665131982822\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.12728175889218607\n",
      "        }\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "}\n",
      "\n",
      "Getting token from http://192.168.39.50:31307/oauth/token\n",
      "{\"access_token\":\"343409a6-0abd-4b8c-ad83-dd1f9657fef4\",\"token_type\":\"bearer\",\"expires_in\":43122,\"scope\":\"read write\"}\n",
      "RECEIVED RESPONSE:\n",
      "meta {\n",
      "  puid: \"6k4ba0l998145lnokbc4t2ncff\"\n",
      "  requestPath {\n",
      "    key: \"complex-model\"\n",
      "    value: \"model-with-metrics-grpc:0.1\"\n",
      "  }\n",
      "  metrics {\n",
      "    key: \"mycounter\"\n",
      "    value: 1.0\n",
      "  }\n",
      "  metrics {\n",
      "    key: \"mygauge\"\n",
      "    type: GAUGE\n",
      "    value: 100.0\n",
      "  }\n",
      "  metrics {\n",
      "    key: \"mytimer\"\n",
      "    type: TIMER\n",
      "    value: 20.200000762939453\n",
      "  }\n",
      "}\n",
      "data {\n",
      "  names: \"t:0\"\n",
      "  names: \"t:1\"\n",
      "  names: \"t:2\"\n",
      "  names: \"t:3\"\n",
      "  ndarray {\n",
      "    values {\n",
      "      list_value {\n",
      "        values {\n",
      "          number_value: 5.018607555163812\n",
      "        }\n",
      "        values {\n",
      "          number_value: 4.150787936306038\n",
      "        }\n",
      "        values {\n",
      "          number_value: 9.145665131982822\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.12728175889218607\n",
      "        }\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!python ../../../util/api_tester/api-tester.py contract.json \\\n",
    "    `minikube ip` `kubectl get svc -l app=seldon-apiserver-container-app -o jsonpath='{.items[0].spec.ports[1].nodePort}'` \\\n",
    "    --oauth-key oauth-key --oauth-secret oauth-secret -p --grpc --oauth-port `kubectl get svc -l app=seldon-apiserver-container-app -o jsonpath='{.items[0].spec.ports[0].nodePort}'`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldondeployment.machinelearning.seldon.io \"mymodel\" deleted\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl delete -f deployment-grpc.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting local Kubernetes cluster...\n",
      "Machine deleted.\n"
     ]
    }
   ],
   "source": [
    "!minikube delete"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
